/* Team 5
 * James McQueen
 * Corwyn Simpson
 * May 1 ish, 2012
 */
package spider.controllers;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.regex.Pattern;

import spider.stores.DataGatherer;
import spider.stores.LinkBuffer;
import spider.stores.PageBuffer;

/**
 * Manages the creation of multiple Parsers. This can be used either as a simple
 * constructor aid for creating Parsers with Parser.Parser(ParserGroup), or run
 * with run() to create its own Parsers for getting the work done.
 * @author Corwyn Simpson
 */
public class ParserGroup implements Runnable
{
    //xkcd.com/208
    //written with help from myregexp.com
    /**
     * The regular expression used to find links within a page. Group 1 is the
     * url contained in the link tag, and Group 2 is the protocol (http, ftp, 
     * etc.)
     */
    public static final String LINK_PATTERN = 
            "<a.*?\\s+href=\\s*\"(([a-z]+:)?[^#\"]+?(?:\\.html?|\\.txt|/))\".*?>";
    /**
     * The number of capturing groups in LINK_PATTERN, including the whole thing
     */
    public static final int EXTRA_GROUP_COUNT = 3;

    /**
     * This is multiplied by an instances my_max_threads to determine how many
     * unfinished parsers to have running when making new parsers. This is so 
     * that when a Parser finishes while this thread is sleeping, there are 
     * still new Parsers that can be run by the thread pool.
     */
    private static final double EXTRA_PARSERS_MULT = 1.2;
    
    /**
     * Creates a new search pattern that will match any link tags or any element
     * of the_keywords. Words that are substrings of other words will still be 
     * matched.
     * @param the_keywords the set of keywords that the Pattern should match to
     * as well as hypertext links.
     */
    public static Pattern buildRegex(final String[] the_keywords)
    {
        String search_regex = LINK_PATTERN;
        for (String key : the_keywords)
        {
            search_regex += "|\\b(\\Q" + key + "\\E)\\b";
        }
        return Pattern.compile(search_regex, Pattern.CASE_INSENSITIVE);
    }

    /**
     * The DataGatherer to report collected data to.
     */
    private DataGatherer my_gatherer;
    /**
     * The LinkBuffer to send all discovered links to. This is what is consumed
     * by the Retriever side of the program.
     */
    private LinkBuffer my_links;
    /**
     * The PageBuffer to get pages that need to be parsed from. This is what is
     * produced by the Retriever side of the program.
     */
    private PageBuffer my_pages;
    /** 
     * The set of keywords that should be searched for and counted.
     */
    private String[] my_keywords;
    /**
     * The regular expression used to search for new links and occurences of 
     * keywords.
     */
    private Pattern my_regex;
    
    //multi-threading overhead variables
    /**
     * The thread pool that executes all the individual Parsers when running 
     * this class with run().
     */
    private ExecutorService my_thread_pool;
    /**
     * The maximum number of threads that this ParserGroup should run at once.
     */
    private int my_max_threads;
    /**
     * The list of Futures used to track the progress of Parser threads that 
     * have been created and added to the thread pool. The size of this list 
     * should not exceed a certain percentage over my_max_threads.
     */
    private List<Future<Parser>> my_future_list;
    /**
     * The RetrieverGroup to conspire with to make sure that Parsing does not
     * stop prematurely.
     */
    private RetrieverGroup my_partner;
    /**
     * Boolean flag used to report when this thread is running.
     */
    private boolean my_running_flag;
        
    /**
     * Creates a new ParserGroup with the given parameters.
     * @param the_gatherer The DataGatherer to record findings to.
     * @param the_links The LinkBuffer to send recorded links to.
     * @param the_pages The PageBuffer to get pages that need to be parsed from.
     * @param the_keywords The list of keywords to count as the page is being 
     * parsed.
     * @param some_max_threads The number of threads to allot to the fixed 
     * thread pool used in run() to process multiple pages at once. 
     */
    public ParserGroup(DataGatherer the_gatherer, LinkBuffer the_links, 
                    PageBuffer the_pages, String[] the_keywords,
                    int some_max_threads)
    {
        my_gatherer = the_gatherer;
        my_links = the_links;
        my_pages = the_pages;
        my_keywords = the_keywords;
        my_regex = buildRegex(the_keywords);
        my_thread_pool = Executors.newFixedThreadPool(some_max_threads);
        my_max_threads = some_max_threads;
        my_future_list = new LinkedList<Future<Parser>>();
    }
 
    /**
     * Returns a reference to the PageBuffer used by this ParserGroup to start
     * Parsers.
     */
    public PageBuffer getPageBuffer()
    {
        return my_pages;
    }
    
    /**
     * Returns the LinkBuffer used by this ParserGroup to put new links into.
     * @see gatherResults(Parser)
     */
    public LinkBuffer getLinkBuffer()
    {
        return my_links;
    }
    
    /**
     * Returns the DataGatherer used by this ParserGroup to record the results
     * of parsing the page.
     * @see gatherResults(Parser)
     */
    public DataGatherer getDataGatherer()
    {
        return my_gatherer;
    }
    
    /**
     * Returns the list of keywords to search for while parsing a page.
     */
    public String[] getKeywords()
    {
        return my_keywords;
    }
    
    /**
     * Returns a Pattern that will match any link tag or keyword. This is 
     * created to avoid the overhead of creating the searching Regex in each 
     * Parser instance.
     * @see ParserGroup.buildRegex(String[])
     */
    public Pattern getSearchRegex()
    {
        return my_regex;
    }
    
    /**
     * Records all the interesting data associated with a Parser to this 
     * ParserGroup's DataGatherer.
     * @param the_parser The parser to retrieve useful data from.
     */
    public void gatherResults(final Parser the_parser)
    {
        my_links.addAllLinks(the_parser.getPageLinks());
        my_gatherer.addAllResults(the_parser.getPageData());
    }
    
    /**
     * Returns true if there are no more Pages in the PageBuffer and the last
     * Parser has finished running. Only useful when this is run as a thread.
     */
    public boolean isDone()
    {
        synchronized (my_future_list)
        {
            return my_future_list.isEmpty() && my_pages.isEmpty();
        }
    }

    /**
     * Sets the RetrieverGroup to poll so that only when both this thread and 
     * the other one are out of work to do will this thread stop running
     */
    public void watchRetrieverGroup(RetrieverGroup the_retriever_group)
    {
        my_partner = the_retriever_group;
    }
    
    /**
     * Does everything parser-related in a highly multi-threaded way.
     */
    public void run()
    {
        my_running_flag = true;
        try
        {
            while (!this.isDone() || !my_partner.isDone())
            {
                //Process all done Parsers and remove them from the list.
                List<Future<Parser>> futures_to_remove = 
                    new ArrayList<Future<Parser>>();
                for (Future<Parser> future : my_future_list)
                {
                    if (future.isDone())
                    {
                        try
                        {
                            gatherResults(future.get());
                        }
                        catch (ExecutionException e)
                        {
                            System.err.println("Parser exception:\n" + e.getCause());
                        }
                        futures_to_remove.add(future);
                    }
                }
                //Actually remove the done Futures now that we aren't iterating
                my_future_list.removeAll(futures_to_remove);
    //TODO: check to see if I should be allotting extra Parsers
                //Add new parsers until we have a few extra beyond my_max_threads
                while (my_future_list.size() < (my_max_threads * EXTRA_PARSERS_MULT)
                                && !my_pages.isEmpty())
                {
                    synchronized (my_future_list)
                    {
                        Parser new_parser = new Parser(this);
                        my_future_list.add(my_thread_pool.submit(new_parser, new_parser));
                    }
                }
                //Sleep for a short little bit to let other threads do things
                Thread.sleep(5);
            }
        }
        catch (InterruptedException e)
        {
            System.err.println("ParserGroup interrupted\n");
        }
        System.out.println("ParserGroup finished running\n");
        my_running_flag = false;
    }

    public boolean isRunning()
    {
        return my_running_flag;
    }

}
